# -*- coding: utf-8 -*-
import scrapy
from csdn.items import CsdnItem
from scrapy.http import Request, FormRequest
import re

class WySpider(scrapy.Spider):
    name = 'wy'
    allowed_domains = ['csdn.net']
    start_urls = ['http://edu.csdn.net/']
    headers={"User-Agent": "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36"}

    def start_requests(self):
        return [Request("https://passport.csdn.net/account/login", meta={"cookiejar": 1}, callback=self.parse)]

    def parse(self, response):
        data={
            "username":"15936342766",
            "password":"Wuyue123.",
        }
        print("登录中")
        return {FormRequest.from_response(response,
                                          meta={"cookiejar": response.meta["cookiejar"]},
                                          headers=self.headers,
                                          formdata=data,
                                          callback=self.next,)}

    def next(self, response):
        yield Request("http://my.csdn.net/", callback=self.next2, meta={"cookiejar": True})

    #登录至个人中心
    def next2(self, response):
        data = response.body
        fh = open("E:\\python\\python练习\\center.html", "wb")
        fh.write(data)
        fh.close()
        print("成功登陆个人中心")
        yield Request("https://edu.csdn.net/courses", callback=self.next3, meta={"cookiejar": True})

    #转到课程页
    def next3(self,response):
        print("正在访问课程中心")
        item=CsdnItem()
        item["courseName"] = response.xpath('//div[@class="titleInfor"]/span/text()').extract()
        item["coursePrice"] = response.xpath('//div[@class="course_dl_list"]//p[@class="clearfix"]/i/text()').extract()

        #去除价格中爬到的空格和换行符
        for i in range(0,len(item["coursePrice"])):
            d=item["coursePrice"][i]
            cp='\n                            (.*?)                        '
            rs=re.compile(cp).findall(d)
            item["coursePrice"][i]=rs[0]

        #爬取课程课时
        ch="<p><em>(.*?)</em>课时"
        data = response.body
        data=data.decode("utf-8")
        rst=re.compile(ch).findall(data)
        item["courseHour"] = rst

        #写入文件

        fh = open("E:\\python\\python练习\\information.txt", "a")
        for i in range(0, len(item["courseName"])):
            datainf={"课程名称":str(item["courseName"][i]),"课程价格":str(item["coursePrice"][i]),"课时数":str(item["courseHour"][i]),}
            fh.write(str(datainf)+'\n')
        fh.close()

        yield item
        for i in range(2,256):
            url="https://edu.csdn.net/courses/k/p"+str(i)
            yield Request(url, callback=self.next3)
